<?php

/**
 * @see Zend_Loader_PluginLoader
 */
require_once 'Zend/Loader/PluginLoader.php';

/**
 * @see Zend_Log
 */
require_once 'Zend/Log.php';

/**
 * Kornak_Http_Crawler_Abstract is an implementation of an HTTP crawler in PHP.
 *
 * @category   Kornak
 * @package    Kornak_Http
 * @subpackage Crawler
 * @throws     Kornak_Http_Crawler_Exception
 * @copyright  Copyright (c) 2007-2009 Kornak Group (http://www.kornak-framework.org)
 * @license    http://www.kornak-framework.org/license/new-bsd     New BSD License
 */
abstract class Kornak_Http_Crawler_Abstract
{
    /**
     * HTTP client. It is used to fetch the pages from the web.
     *
     * @var Zend_Http_Client
     */
    protected $_httpClient;

    /**
     * Message logger. It is used to log messages to various locations.
     *
     * @var Zend_Log
     */
    protected $_logger;

    /**
     * Crawler plugin loader. It is used to load the crawler plugins classes.
     *
     * @var Zend_Loader_PluginLoader
     */
    protected $_pluginLoader;

    /**
     * Crawler plugins instances.
     *
     * @var array
     */
    protected $_plugins = array();


    /**
     * Http response for the last HTTP client request.
     *
     * @var Zend_Http_Response
     */
    protected $_response;

    /**
     * Crawl scheduler. Its purpose is to manage the queue of pages to be
     * crawled.
     *
     * @var Kornak_Crawler_Scheduler
     */
    protected $_scheduler;

    /**
     * URL of the last HTTL client request.
     *
     * @var string
     */
    protected $_url;

    /**
     * This method is used to initialize the crawler and is called at the end
     * of the contructor. The default implementation does nothing.
     * It could be overriden by inherited classes if needed.
     *
     * @return void
     */
    protected function _init()
    {
        // Does nothing on purpose
    }

    /**
     * Process the HTTP response. Here, the crawler can do anything needed such
     * as extracting data from response, storing data to disk, etc.
     *
     * @param $url      string              URL for the curent page
     * @param $response Zend_Http_Response  HTTP response for the current page
     * @return void
     */
    abstract protected function _processResponse($url, Zend_Http_Response $response);

    /**
     * Called when crawling starts. It could be overriden to take action when
     * the crawling starts.
     *
     * @return void
     */
    protected function _startCrawling()
    {
        // Does nothing on purpose
    }

    /**
     * Called when crawling stops. It could be overriden to take action when
     * the crawling stops.
     *
     * @return void
     */
    protected function _stopCrawling()
    {
        // Does nothing on purpose
    }

    /**
     * Constructor method. Sets up the plugin loader and call the _init() method
     * at its end.
     *
     * @return void
     */
    public function __construct()
    {
        $this->_pluginLoader = new Zend_Loader_PluginLoader();
        $this->_pluginLoader->addPrefixPath('Kornak_Http_Crawler_Plugin_', 'Kornak/Http/Crawler/Plugin/');

        $this->_init();
    }

    /**
     * Returns HTTP client. If none is defined at the time, it instanciates a
     * default one.
     *
     * @return Zend_Http_Client
     */
    public function getHttpClient()
    {
        if (null == $this->_httpClient) {
            require_once 'Zend/Http/Client.php';
            $config = array('useragent' => 'Kornak based crawler');
            $this->_httpClient = new Zend_Http_Client(null, $config);
        }

        return $this->_httpClient;
    }

    /**
     * Returns message logger. If none is defined at the time, it instanciates a
     * default one which outputs messages.
     *
     * @return Zend_Log
     */
    public function getLogger()
    {
        if (null == $this->_logger) {
            require_once 'Zend/Log.php';
            $this->_logger = new Zend_Log();

            require_once 'Zend/Log/Writer/Stream.php';
            $this->_logger->addWriter(
                new Zend_Log_Writer_Stream('php://output')
            );
        }

        return $this->_logger;
    }

    /**
     * Returns a plugin by its name. If the plugin is not loaded yet, it will
     * load it using the plugin loader before returning it.
     *
     * @var    $name    string  Name of the plugin to retrieve
     * @return Kornak_Http_Crawler_Plugin_Abstract
     */
    public function getPlugin($name)
    {
        if (!array_key_exists($name, $this->_plugins)) {
            $class = $this->_pluginLoader->load($name);
            $this->_plugins[$name] = new $class($this);
        }

        $plugin = $this->_plugins[$name];
        $plugin->setUrl($this->_url);
        $plugin->setResponse($this->_response);

        return $plugin;
    }

    /**
     * Returns scheduler. If none is defined at the time, it instanciates a
     * default one.
     *
     * @return Kornak_Http_Crawler_Scheduler
     */
    public function getScheduler()
    {
        if (null == $this->_scheduler) {
            require_once 'Kornak/Http/Crawler/Scheduler.php';
            $this->_scheduler = new Kornak_Http_Crawler_Scheduler($this);
        }

        return $this->_scheduler;
    }

    /**
     * Logs messages using the message logger.
     *
     * @param $message  string  Message to log
     * @param $level    int     Message level
     * @return void
     */
    public function log($message, $level = Zend_Log::INFO)
    {
        $this->getLogger()->log($message, $level);
    }

    /**
     * This is the main method of the crawler. It manages the crawling.
     *
     * @return void
     */
    public function run()
    {
        $this->log('Start crawling', Zend_Log::DEBUG);
        $this->_startCrawling();

        $scheduler = $this->getScheduler();
        while ($url = $scheduler->getNextUrl()) {
            $this->log("Processing '$url'", Zend_Log::DEBUG);

            $response = $this->getHttpClient()
                             ->resetParameters()
                             ->setUri($url)
                             ->request();

            $this->_url      = $url;
            $this->_response = $response;

            $links = $this->getPlugin('ExtractLinks')->getData();
            $scheduler->addUrl($links);

            $this->_processResponse($url, $response);

            $scheduler->setUrlStatus($url, Kornak_Http_Crawler_Scheduler::URL_STATUS_DONE);
        }

        $this->log('Stop crawling', Zend_Log::DEBUG);
        $this->_stopCrawling();
    }

    /**
     * Sets the HTTP client that will be used by the crawler.
     *
     * @param $httpClient   Zend_Http_Client    HTTP client to use
     * @return Kornak_Http_Crawler_Abstract
     */
    public function setHttpClient(Zend_Http_Client $httpClient)
    {
        $this->_httpClient = $httpClient;
        return $this;
    }

    /**
     * Sets the message logger that will be used by the crawler.
     *
     * @param $logger   Zend_Log    Message logger to use
     * @return Kornak_Http_Crawler_Abstract
     */
    public function setLogger(Zend_Log $logger)
    {
        $this->_logger = $logger;
        return $this;
    }

    /**
     * Sets the scheduler that will be used by the crawler.
     *
     * @param $scheduler    Kornak_Http_Crawler_Scheduler_Abstract   Scheduler to use
     * @return Kornak_Http_Crawler_Abstract
     */
    public function setScheduler(Kornak_Http_Crawler_Scheduler_Abstract $scheduler)
    {
        $scheduler->setCrawler($this);
        $this->_scheduler = $scheduler;
        return $this;
    }

    /**
     * Sets HTTP client user agent string.
     *
     * @param $userAgent    string  User agent string to sue
     * @return Kornak_Http_Crawler_Abstract
     */
    public function setUserAgent($userAgent)
    {
        $this->getHttpClient()->setConfig(array('useragent' => $userAgent));
        return $this;
    }
}